﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

namespace ReBasic.Helper
{
    public class WikiFormatHelper
    {
        //Goal: Generate Sent_ds, Term_ds, Chunk.

        StreamReader sr;
        StreamWriter sw;
        FileStream fsr, fsw;
        const string wikiPath = MainFrame.DataPath + @"Wikipedia Dump\";

        string[] workFile = { "Sent_ds.txt", "Sent.txt", "Sent_Naive.txt", "Term_ds.txt" };

        int tot_cnt;

        public event MainFrame.TripleHandler ProcessEvent;
        public event MainFrame.InfoHandler StateEvent;

        void GenerateNavieSent()
        {
            int sent_process = 0;
            int local_cnt = 0;
            StringBuilder sb;
            int i, len;
            StateEvent("Start generating SentNaive...", false);
            while (!sr.EndOfStream)
            {
                sent_process++;
                string rd = sr.ReadLine();
                string[] spt = rd.Split(new string[] { "\t\t" }, StringSplitOptions.RemoveEmptyEntries);
                if (spt.Length < 4) continue;
                string[] tags = spt[2].Split(' ');

                for (i = 0, len = tags.Length, sb = new StringBuilder(); i < len; i++)
                {
                    int idx = tags[i].LastIndexOf('/');
                    string wd = tags[i].Substring(0, idx).ToLower();
                    sb.Append(' ').Append(wd);
                }
                if (sb.ToString().Contains("is a village in the administrative district of"))
                    continue;
                else
                {
                    local_cnt++;
                    sw.WriteLine(sb.ToString());
                }

                if (sent_process % 10000 == 0)
                    ProcessEvent(sent_process, local_cnt, 0);
            }
            tot_cnt += local_cnt;
            StateEvent("Generate with " + local_cnt.ToString() + " sentences, total = " + tot_cnt.ToString(), true);
        }

        void GenerateSent(bool isDollarSeparate)
        {
            int sent_process = 0;
            int local_cnt = 0;
            StringBuilder sb;
            int i, len, flag = 0;
            List<string> pool;
            if (!isDollarSeparate)
                StateEvent("Start generating Sent...", false);
            else
                StateEvent("Start generating SentDs...", false);
            while (!sr.EndOfStream)
            {
                sent_process++;
                string rd = sr.ReadLine();
                string[] spt = rd.Split(new string[] { "\t\t" }, StringSplitOptions.RemoveEmptyEntries);
                if (spt.Length < 4) continue;
                string[] tags = spt[2].Split(' ');

                #region Remove Some Noisy Sentences

                //for (i = 0, len = tags.Length, sb = new StringBuilder(); i < len; i++)
                //{
                //    int idx = tags[i].LastIndexOf('/');
                //    string wd = tags[i].Substring(0, idx).ToLower();
                //    sb.Append(' ').Append(wd);
                //}
                //if (sb.ToString().Contains("is a village in the administrative district of"))
                //    continue;

                #endregion
                
                #region Generate Sentences From Tags
                
                pool = new List<string>();
                flag = 0;
                sb = new StringBuilder();
                for (i = 0, len = tags.Length; i < len; i++)
                {
                    #region Split sentences
                    int idx = tags[i].LastIndexOf('/');
                    string wd = tags[i].Substring(0, idx).ToLower();
                    if (wd == ".")
                    {
                        if (sb.ToString() != "")
                            pool.Add(sb.ToString());
                        flag = 0;
                        sb = new StringBuilder();
                    }
                    else
                    { 
                        if (flag++ > 0)
                            sb.Append(' ');
                        if (isDollarSeparate && wd.Length == 1)
                            sb.Append('$');
                        else
                            sb.Append(wd);
                    }
                    #endregion
                }
                if (sb.ToString() != "")
                    pool.Add(sb.ToString());
                
                #region Output

                foreach (string str in pool)
                    sw.WriteLine(str);
                local_cnt += pool.Count;

                #endregion
                
                #endregion

                if (sent_process % 10000 == 0)
                    ProcessEvent(sent_process, local_cnt, 0);
            }
            tot_cnt += local_cnt;
            StateEvent("Generate with " + local_cnt.ToString() + " sentences, total = " + tot_cnt.ToString(), true);
        }

        public void Work(int workMode)
        {
            tot_cnt = 0;
            fsw = new FileStream(
                MainFrame.DataPath + workFile[workMode], 
                FileMode.Create);
            sw = new StreamWriter(fsw, Encoding.UTF8);
            DirectoryInfo dir = new DirectoryInfo(wikiPath);
            foreach (FileInfo fi in dir.GetFiles())
            {
                fsr = new FileStream(fi.FullName, FileMode.Open);
                sr = new StreamReader(fsr, Encoding.UTF8);
                if (workMode == 0)
                    GenerateSent(true);
                else if (workMode == 1)
                    GenerateSent(false);
                else if (workMode == 2)
                    GenerateNavieSent();
                sr.Close();
                fsr.Close();
            }
            StateEvent("Total Sentences: " + tot_cnt.ToString() + ".", true);
            sw.Close();
            fsw.Close();
        }

        public void CloseFile(int whichWork)
        {
            if (sr != null)
            {
                sr.Close();
                fsr.Close();
            }
            if (sw != null)
            {
                sw.Close();
                fsw.Close();
            }
            if (whichWork == 0)         //Gen Sent_ds
                StateEvent("Gen Sent_ds stopped, with "
                    + tot_cnt.ToString() + " sentences.", true);
            if (whichWork == 1)         //Gen Sent
                StateEvent("Gen Sent stopped, with "
                    + tot_cnt.ToString() + " sentences.", true);
        }
    }
}
